/*
 * Copyright (c) 2015, 2025, Oracle and/or its affiliates.
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License, version 2.0,
 * as published by the Free Software Foundation.
 *
 * This program is designed to work with certain software (including
 * but not limited to OpenSSL) that is licensed under separate terms,
 * as designated in a particular file or component or in included license
 * documentation.  The authors of MySQL hereby grant you an additional
 * permission to link the program and your derivative works with the
 * separately licensed software that they have either included with
 * the program or referenced in the documentation.
 *
 * This program is distributed in the hope that it will be useful,  but
 * WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See
 * the GNU General Public License, version 2.0, for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software Foundation, Inc.,
 * 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
 */

#include "unittest/shell_script_tester.h"

#include <mysql_version.h>
#include <chrono>
#include <condition_variable>
#include <iostream>
#include <set>
#include <string>
#include <thread>
#include <utility>

#include "modules/adminapi/common/server_features.h"
#include "mysqlshdk/libs/textui/textui.h"
#include "mysqlshdk/shellcore/shell_console.h"
#include "shellcore/interrupt_handler.h"
#include "shellcore/ishell_core.h"
#include "src/mysqlsh/cmdline_shell.h"
#include "utils/process_launcher.h"
#include "utils/utils_file.h"
#include "utils/utils_general.h"
#include "utils/utils_lexing.h"
#include "utils/utils_path.h"
#include "utils/utils_string.h"
#include "utils/version.h"

using namespace shcore;

extern "C" const char *g_test_home;
extern bool g_generate_validation_file;
extern int g_test_script_timeout;
extern std::set<int> g_break;

// Helper class to capture GTest partial results (the stuff generated by
// ADD_TEST_FAILURE() and so on)
class TestResultCatcher : public ::testing::EmptyTestEventListener {
 public:
  TestResultCatcher(std::stringstream *stream) : m_stream(stream) {}

  void OnTestPartResult(const ::testing::TestPartResult &result) override {
    // If the test part succeeded, we don't need to do anything.
    if (result.type() == ::testing::TestPartResult::kSuccess) return;

    *m_stream << "\033[35m\033[1m################################### vv BEGIN "
                 "FAILURE vv ###################################\033[0m\n";
    *m_stream << result.message() << "\n";
    *m_stream << "\033[35m\033[1m################################### ^^ END "
                 "FAILURE ^^ ###################################\033[0m\n";
  }

 private:
  std::stringstream *m_stream;
};

//-----------------------------------------------------------------------------

static bool do_print(void * /* udata */, const char *s) {
  printf("%s", s);
  fflush(stdout);
  return true;
}

static shcore::Prompt_result do_prompt(
    void * /* udata */, const char *prompt,
    const shcore::prompt::Prompt_options & /*options*/,
    std::string *ret_input) {
  printf("%s", prompt);
  fflush(stdout);
  std::cin >> *ret_input;
  return shcore::Prompt_result::Ok;
}

namespace mysqlsh {
class Test_debugger {
 public:
  enum class Action { Skip_execute, Continue, Abort };

  Test_debugger() : m_deleg(this, do_print, do_prompt, nullptr, nullptr) {
    m_console.reset(new mysqlsh::Shell_console(&m_deleg));
  }

  void enable(bool step) {
    println("Enabling...");
    m_enabled = true;
    m_stepping = step;
  }

  void on_test_begin(Shell_core_test_wrapper *test) { m_test = test; }

  void on_reset_shell(std::shared_ptr<mysqlsh::Command_line_shell> shell) {
    m_shell = shell;
  }

  Action will_execute(const std::string &source, int lnum,
                      std::string_view code) {
    if (m_main_source.empty()) m_main_source = source;

    // Line containing //BREAK in the script will enter cmd loop
    if (!m_enabled) return Action::Continue;

    if (m_stepping) {
      if (source != m_main_source && m_skip_include) {
        // skip over include files
        return Action::Continue;
      }
      m_stepping = false;
      return interact();
    }

    if (auto code_stripped = shcore::str_strip_view(code);
        (code_stripped == "//BREAK") || (code_stripped == "#BREAK")) {
      println("//BREAK hit");
      return interact();
    }

    if (source == m_main_source && g_break.count(lnum) > 0) {
      println("Breaking at requested line " + std::to_string(lnum));
      return interact();
    }

    return Action::Continue;
  }

  Action on_validate_fail(const std::string &chunk_id) {
    if (m_exit_on_test_error) exit(1);
    if (m_enabled) {
      println("Validation '" + chunk_id + "' failed");
      return interact(true);
    } else if (g_test_fail_early) {
      return Action::Abort;
    }

    println(
        "\033[35m\033[1m#######################################################"
        "##############################\033[0m");

    return Action::Continue;
  }

  void did_execute(int /* lnum */, const std::string & /* code */) {}

  Action did_execute_test_failure() {
    if (m_exit_on_test_error) exit(1);
    if (m_enabled) {
      println("Test failures found");
      return interact();
    } else if (g_test_fail_early) {
      return Action::Abort;
    }
    return Action::Continue;
  }

  void did_throw(int /* lnum */, const std::string & /* code */) {
    if (m_enabled && m_break_on_throw) interact();
  }

 private:
  void println(const std::string &s) {
    puts(mysqlshdk::textui::bold("TDB: " + s).c_str());
  }

  bool debug() { return true; }

  bool handle_command(const std::string &cmd, bool *exit_interactive) {
    if (cmd == "\\next") {
      m_stepping = true;
      m_skip_include = true;
      *exit_interactive = true;
      return true;
    } else if (cmd == "\\step") {
      m_stepping = true;
      m_skip_include = false;
      *exit_interactive = true;
      return true;
    } else if (cmd == "\\dhelp") {
      std::cout << "TDB Help\n"
                << "--------\n"
                << "\\next line (skip INCLUDE)\n"
                << "\\step line (enter INCLUDE)\n"
                << "\\cont continue execution\n"
                << "\\abort let test fail (when handling validation failures)\n"
                << "\\quit exit\n"
                << "\n";
      return true;
    }
    return false;
  }

  Action interact(bool abortable = false) {
    if (m_first_interact) {
      if (abortable)
        println(
            "Entering interactive loop... \\cont to continue, ^D or \\abort to "
            "exit. \\dhelp for debugger help");
      else
        println(
            "Entering interactive loop... \\cont or ^D to continue. \\dhelp "
            "for debugger help");
      m_first_interact = false;
    }

    // save stdout and stderr contents
    std::string std_err = m_test->output_handler.std_err;
    std::string std_out = m_test->output_handler.std_out;

    Action result = do_interact(abortable);
    if (result == Action::Continue)
      println("Continuing...");
    else
      println("Aborting...");

    m_test->output_handler.std_err = std::move(std_err);
    m_test->output_handler.std_out = std::move(std_out);

    return result;
  }

#define CTRL_C_STR "\003"
  Action do_interact(bool abortable = false) {
    Action result = abortable ? Action::Abort : Action::Continue;
    bool interrupted = false;
    bool exit_interactive = false;
    std::string cmd;
    while (!exit_interactive) {
      char *tmp = mysqlsh::Command_line_shell::readline(
          makebold(shell()->prompt()).c_str());
      if (tmp && strcmp(tmp, CTRL_C_STR) != 0) {
        cmd = tmp;
        free(tmp);
      } else {
        if (tmp) {
          if (strcmp(tmp, CTRL_C_STR) == 0) interrupted = true;
          free(tmp);
        }
        if (interrupted) {
          shell()->clear_input();
          interrupted = false;
          continue;
        }
        break;
      }

      if (cmd == "") {
        // re-execute last command
        cmd = m_last_cmd;
      } else {
        m_last_cmd = cmd;
      }

      if (cmd == "\\cont") {
        result = Action::Continue;
        break;
      } else if (cmd == "\\abort" && abortable) {
        result = Action::Abort;
        break;
      } else if (cmd == "\\quit") {
        exit(1);
      }

      if (!handle_command(cmd, &exit_interactive)) shell()->process_line(cmd);
    }
    return result;
  }

 private:
  bool m_enabled = false;
  bool m_break_on_throw = false;
  bool m_stepping = false;
  bool m_skip_include = false;
  bool m_exit_on_test_error = false;
  bool m_first_interact = true;
  Shell_core_test_wrapper *m_test = nullptr;
  std::string m_last_cmd;
  std::string m_main_source;

  std::shared_ptr<mysqlsh::Shell_console> m_console;
  shcore::Interpreter_delegate m_deleg;
  std::weak_ptr<mysqlsh::Command_line_shell> m_shell;

  std::shared_ptr<mysqlsh::Command_line_shell> shell() {
    return m_shell.lock();
  }
};
}  // namespace mysqlsh

mysqlsh::Test_debugger *g_tdb = nullptr;

void init_tdb() {
  if (!g_tdb) g_tdb = new mysqlsh::Test_debugger();
}

void enable_tdb(bool step) {
  init_tdb();
  g_tdb->enable(step);
}

void fini_tdb() {
  delete g_tdb;
  g_tdb = nullptr;
}

//-----------------------------------------------------------------------------

class Timeout {
 public:
  explicit Timeout(Shell_script_tester *owner) : _owner(owner) { start(); }

  ~Timeout() { stop(); }

  bool did_timeout() const { return _timed_out; }

 private:
  Shell_script_tester *_owner;
  std::thread _thread;
  std::condition_variable _wait_cond;
  std::mutex _cond_mutex;
  bool _stop = false;
  bool _timed_out = false;

  void check() {
    using namespace std::chrono_literals;  // NOLINT:build/namespace

    // wait for the timeout amount then abort the test
    std::unique_lock<std::mutex> lock(_cond_mutex);
    if (!_wait_cond.wait_for(lock, g_test_script_timeout * 1s,
                             [this]() { return _stop; })) {
      _timed_out = true;
      // timedout
      puts("TIMEOUT!");

      // there's no locking to protect this, but the test already timedout
      // anyway
      _owner->output_handler.flush_debug_log();

      puts("Aborting test after timeout");
      auto intr = shcore::current_interrupt(true);
      if (intr) intr->interrupt();
    }
  }

  void start() {
    _stop = false;
    _thread = std::thread([this]() { check(); });
  }

  void stop() {
    {
      std::unique_lock<std::mutex> lock(_cond_mutex);
      if (_stop) return;
      _stop = true;
    }
    _wait_cond.notify_all();
    _thread.join();
  }
};

//-----------------------------------------------------------------------------

Shell_script_tester::Shell_script_tester() {
  init_tdb();

  // Default home folder for scripts
  _shell_scripts_home = shcore::path::join_path(g_test_home, "scripts");
  _new_format = false;
}

void Shell_script_tester::SetUp() {
  Crud_test_wrapper::SetUp();

  if (_options->trace_protocol) {
    // Redirect cout
    _cout_backup = std::cout.rdbuf();
    std::cout.rdbuf(_cout.rdbuf());
  }

  g_tdb->on_test_begin(this);
}

void Shell_script_tester::TearDown() {
  if (!_skip_sandbox_check) {
    // check for leftover sandboxes
    for (int i = 0; i < tests::sandbox::k_num_ports; i++) {
      EXPECT_FALSE(shcore::is_folder(
          testutil->get_sandbox_path(_mysql_sandbox_ports[i])))
          << "Sandbox left behind port=" << _mysql_sandbox_ports[i];
    }
  }

  Crud_test_wrapper::TearDown();

  if (_options->trace_protocol) {
    // Restore old cout.
    std::cout.rdbuf(_cout_backup);
  }

  g_tdb->on_test_begin(this);
}

void Shell_script_tester::reset_shell() {
  Crud_test_wrapper::reset_shell();
  g_tdb->on_reset_shell(_interactive_shell);
}

void Shell_script_tester::set_config_folder(const std::string &name) {
  // Custom home folder for scripts
  _shell_scripts_home = shcore::path::join_path(g_test_home, "scripts", name);

  // Currently hardcoded since scripts are on the shell repo
  // but can easily be updated to be setup on an ENV VAR so
  // the scripts can by dynamically imported from the dev-api docs
  // with the sctract tool Jan is working on
  _scripts_home = shcore::path::join_path(_shell_scripts_home, "scripts");
}

void Shell_script_tester::set_setup_script(const std::string &name) {
  // if name is an absolute path, join_path will just return name
  _setup_script = shcore::path::join_path(_shell_scripts_home, "setup", name);
}

std::optional<std::string> Shell_script_tester::resolve_token(
    const std::string &token) {
  if (auto resolve = Shell_test_env::resolve_token(token);
      resolve.has_value()) {
    return resolve;
  }

  // we use whatever is defined on the scripting language
  output_handler.internal_std_out.clear();
  execute_internal(token);
  auto value = std::exchange(output_handler.internal_std_out, {});

  // strip the trailing newline added by execute
  if (str_endswith(value, "\n")) value.pop_back();

  return value;
}

std::string Shell_script_tester::resolve_string(std::string source) {
  // optimization: almost all source strings don't need a resolve, so catch that
  // as soon as we can
  if (source.find('<') == std::string::npos) return source;

  auto std_out_backup = std::exchange(output_handler.std_out, {});

  auto find_token = [](std::string_view code, std::string_view find,
                       std::string_view is_not, size_t start_pos) -> size_t {
    while (true) {
      auto found = code.find(find, start_pos);
      if (found == std::string_view::npos) return found;

      auto proto = code.find(is_not, start_pos);
      if (found != proto) return found;

      start_pos = proto + is_not.size();
    }

    // std::unreachable();
    return std::string_view::npos;
  };

  size_t start = find_token(source, "<<<", "<<<< RECEIVE", 0);
  size_t end;

  while (start != std::string::npos) {
    bool strip_trailing_newline = false;

    end = find_token(source, ">>>", ">>>> SEND", start);
    if (end == std::string::npos)
      throw std::logic_error("Unterminated <<< in test");

    //<<<"fooo"\>>>\n is stripped into fooo (without the trailing newline)
    if (source.compare(end - 1, 5, "\\>>>\n") == 0) {
      strip_trailing_newline = true;
      --end;
    }

    auto value = resolve_token(source.substr(start + 3, end - start - 3));
    assert(value.has_value());

    if (strip_trailing_newline) {
      source.replace(start, end - start + 5, *value);
    } else {
      source.replace(start, end - start + 3, *value);
    }

    start = find_token(source, "<<<", "<<<< RECEIVE", 0);
  }

  output_handler.std_out = std::move(std_out_backup);
  return source;
}

bool Shell_script_tester::validate_line_by_line(const std::string &context,
                                                const std::string &chunk_id,
                                                const std::string &stream,
                                                const std::string &expected,
                                                const std::string &actual,
                                                int srcline, int valline) {
  std::vector<std::string> expected_lines;
  bool changed = false;
  // Takes this as the posibility of the presense of conditional lines.
  if (expected.find("?{}") != std::string::npos) {
    expected_lines = shcore::split_string(expected, "\n");

    // Lines in the format of ?{<condition>} might be the start/end of a
    // conditional Section of expectations, the section ends on a equal to ?{}
    for (size_t index = 0; index < expected_lines.size(); index++) {
      const auto &line = expected_lines[index];
      if (line.empty() || line[0] != '?') continue;

      auto size = line.size();
      if (!(size > 2 && line[1] == '{' && line[2] != '*' &&
            line[size - 1] == '}'))
        continue;

      // Empty condition is simply ignored
      auto condition = line.substr(2, size - 3);
      if (condition.empty()) continue;

      expected_lines.erase(expected_lines.begin() + index);

      // If condition not satisfied deletes all the lines in the middle
      // until the closing tag is found
      bool erase = !context_enabled(std::move(condition));
      while (expected_lines.size() > index && expected_lines[index] != "?{}") {
        if (erase)
          expected_lines.erase(expected_lines.begin() + index);
        else
          index++;
      }

      expected_lines.erase(expected_lines.begin() + index);

      changed = true;

      // Goes back on the index so the new current line is analyzed as
      // well
      index--;
    }
  }

  return check_multiline_expect(
      shcore::str_format("%s@%s", context.c_str(), chunk_id.c_str()), stream,
      changed ? shcore::str_join(expected_lines, "\n") : expected, actual,
      srcline, valline);
}

bool Shell_script_tester::validate(const std::string &context,
                                   const std::string &chunk_id, bool optional) {
  std::string original_std_out = output_handler.std_out;
  std::string original_std_err = output_handler.std_err;

  size_t out_position = 0;
  size_t err_position = 0;

  const std::string &validation_id = _chunks[chunk_id].def.validation_id;

  if (m_chunk_validations.find(validation_id) == m_chunk_validations.end()) {
    // There were errors
    if (!original_std_err.empty()) {
      ADD_FAILURE_AT(_chunks[chunk_id].source.c_str(),
                     _chunks[chunk_id].code[0].first)
          << "while executing chunk: " + _chunks[chunk_id].def.line << "\n"
          << makered("\tUnexpected Error: ") + original_std_err << "\n";
    } else if (!optional && _chunks.find(chunk_id) != _chunks.end()) {
      // The error is that there are no validations
      ADD_FAILURE_AT(_chunks[chunk_id].source.c_str(),
                     _chunks[chunk_id].code[0].first)
          << makered("MISSING VALIDATIONS FOR CHUNK ")
          << _chunks[chunk_id].def.line << "\n"
          << makeyellow("\tSTDOUT: ") << original_std_out << "\n"
          << makeyellow("\tSTDERR: ") << original_std_err << "\n";
    }
    output_handler.wipe_all();
    _cout.str("");
    _cout.clear();

    return true;
  }

  bool expect_failures = false;

  // Identifies the validations to be done based on the context
  std::vector<std::reference_wrapper<const Validation>> validations;
  for (const auto &val : m_chunk_validations[validation_id]) {
    bool enabled = false;
    try {
      enabled = context_enabled(val.def.context);

      if (val.def.stream == "PROTOCOL" && !_options->trace_protocol) {
        ADD_FAILURE_AT("validation file", val.def.linenum)
            << "ERROR TESTING PROTOCOL: Protocol tracing is disabled."
            << "\n"
            << "\tCHUNK: " << val.def.line << "\n";
      }
    } catch (const std::invalid_argument &e) {
      ADD_FAILURE_AT("validation file", val.def.linenum)
          << "ERROR EVALUATING VALIDATION CONTEXT: " << e.what() << "\n"
          << "\tCHUNK: " << val.def.line << "\n";
    }

    if (enabled) {
      validations.push_back(std::cref(val));

      if (!val.expected_error.empty()) expect_failures = true;
    } else {
      if (output_handler.internal_std_err.empty()) {
        SKIP_VALIDATION(val.def.line);
      } else {
        SKIP_VALIDATION(val.def.line + ": " + output_handler.internal_std_err);
      }
    }
  }

  std::string full_statement;
  bool first_validation{true};
  // The validations will be performed ONLY if the context is enabled
  for (const Validation &val : validations) {
    // Validation goes against validation code
    if (!val.code.empty()) {
      // Before cleaning up, prints any error found on the script execution
      if (first_validation && !original_std_err.empty()) {
        ADD_FAILURE_AT(_chunks[chunk_id].source.c_str(),
                       _chunks[chunk_id].code[0].first)
            << makered("\tUnexpected Error: " + original_std_err) << "\n";
        output_handler.wipe_all();
        return false;
      }

      output_handler.wipe_all();
      _cout.str("");
      _cout.clear();

      {
        auto backup = _custom_context;
        full_statement.append(val.code);
        _custom_context.append(1, '[').append(full_statement).append(1, ']');
        execute(val.code);
        _custom_context = std::move(backup);
      }

      if (_interactive_shell->input_state() == shcore::Input_state::Ok)
        full_statement.clear();
      else
        full_statement.append("\n");

      original_std_err = output_handler.std_err;
      original_std_out = output_handler.std_out;

      out_position = 0;
      err_position = 0;

      output_handler.wipe_all();
      _cout.str("");
      _cout.clear();
    }

    first_validation = false;

    // Validates unexpected error
    if (!expect_failures && !original_std_err.empty()) {
      ADD_FAILURE_AT(_chunks[chunk_id].source.c_str(),
                     _chunks[chunk_id].code[0].first)
          << "while executing chunk: " + _chunks[chunk_id].def.line << "\n"
          << makered("\tUnexpected Error: ") << original_std_err << "\n";
      output_handler.wipe_all();
      return false;
    }

    // Validates expected output if any
    if (!val.expected_output.empty()) {
      auto out = resolve_string(val.expected_output);

      if (out != "*") {
        if (val.def.validation == ValidationType::Simple) {
          auto matched =
              multi_value_compare(out, original_std_out, false, out_position,
                                  nullptr, &out_position);
          if (!matched) {
            if (out_position == 0) {
              ADD_FAILURE_AT(_chunks[chunk_id].source.c_str(),
                             _chunks[chunk_id].code[0].first)
                  << "while executing chunk: " + _chunks[chunk_id].def.line
                  << "\nwith validation at " << val.def.linenum << "\n"
                  << makeyellow("\tSTDOUT missing: ") << out << "\n"
                  << makeyellow("\tSTDOUT actual: ") + original_std_out << "\n";
            } else {
              ADD_FAILURE_AT(_chunks[chunk_id].source.c_str(),
                             _chunks[chunk_id].code[0].first)
                  << "while executing chunk: " + _chunks[chunk_id].def.line
                  << "\nwith validation at " << val.def.linenum << "\n"
                  << makeyellow("\tSTDOUT missing: ") << out << "\n"
                  << makeyellow("\tSTDOUT actual: ") +
                         original_std_out.substr(out_position)
                  << "\n"
                  << makeyellow("\tSTDOUT original: ") + original_std_out
                  << "\n";
            }
            output_handler.wipe_all();
            return false;
          }
        } else {
          SCOPED_TRACE(_chunks[chunk_id].source);
          if (!validate_line_by_line(
                  context, chunk_id, "STDOUT", out,
                  val.def.stream == "PROTOCOL" ? _cout.str() : original_std_out,
                  _chunks[chunk_id].code[0].first, val.def.linenum)) {
            output_handler.wipe_all();
            return false;
          }
        }
      }
    }

    // Validates unexpected output if any
    if (!val.unexpected_output.empty()) {
      auto out = resolve_string(val.unexpected_output);
      bool matched = false;
      if (val.def.stream == "PROTOCOL")
        matched = _cout.str().find(out) != std::string::npos;
      else
        matched = multi_value_compare(out, original_std_out, false);

      if (matched) {
        ADD_FAILURE_AT(_chunks[chunk_id].source.c_str(),
                       _chunks[chunk_id].code[0].first)
            << "while executing chunk: " + _chunks[chunk_id].def.line << "\n"
            << "with validation at " << val.def.linenum << "\n"
            << makeyellow("\tSTDOUT unexpected: ") << out << "\n"
            << makeyellow("\tSTDOUT actual: ") << original_std_out << "\n";
        output_handler.wipe_all();
        return false;
      }
    }

    // Validates expected error if any
    if (!val.expected_error.empty()) {
      auto error = resolve_string(val.expected_error);

      if (error != "*") {
        if (val.def.validation == ValidationType::Simple) {
          bool matched =
              multi_value_compare(error, original_std_err, false, err_position,
                                  nullptr, &err_position);
          if (!matched) {
            if (err_position == 0) {
              ADD_FAILURE_AT(_chunks[chunk_id].source.c_str(),
                             _chunks[chunk_id].code[0].first)
                  << "while executing chunk: " + _chunks[chunk_id].def.line
                  << "\n"
                  << "with validation at " << val.def.linenum << "\n"
                  << makeyellow("\tSTDERR missing: ") + error << "\n"
                  << makeyellow("\tSTDERR actual: ") + original_std_err << "\n";
            } else {
              ADD_FAILURE_AT(_chunks[chunk_id].source.c_str(),
                             _chunks[chunk_id].code[0].first)
                  << "while executing chunk: " + _chunks[chunk_id].def.line
                  << "\n"
                  << "with validation at " << val.def.linenum << "\n"
                  << makeyellow("\tSTDERR missing: ") + error << "\n"
                  << makeyellow("\tSTDERR actual: ") +
                         original_std_err.substr(err_position)
                  << "\n"
                  << makeyellow("\tSTDERR original: ") + original_std_err
                  << "\n";
            }
            output_handler.wipe_all();
            return false;
          }
        } else {
          SCOPED_TRACE(_chunks[chunk_id].source);
          if (!validate_line_by_line(
                  context, chunk_id, "STDERR", error, original_std_err,
                  _chunks[chunk_id].code[0].first, val.def.linenum)) {
            output_handler.wipe_all();
            return false;
          }
        }
      }
    }
  }

  if (!optional && validations.empty()) {
    ADD_FAILURE_AT(_chunks[chunk_id].source.c_str(),
                   _chunks[chunk_id].code[0].first)
        << makered("MISSING VALIDATIONS FOR CHUNK ")
        << _chunks[chunk_id].def.line << "\n"
        << makeyellow("\tSTDOUT: ") << original_std_out << "\n"
        << makeyellow("\tSTDERR: ") << original_std_err << "\n";
    return false;
  }
  output_handler.wipe_all();
  _cout.str("");
  _cout.clear();

  return true;
}

void Shell_script_tester::validate_interactive(const std::string &script) {
  _filename = script;
  try {
    if (output_handler.passwords.size() || output_handler.prompts.size()) {
      std::cerr << "Starting with " << output_handler.passwords.size()
                << " password prompts and " << output_handler.prompts.size()
                << " regular prompts queued\n";
    }
    execute_script(script, true);
    if (testutil->test_skipped()) {
      SKIP_TEST(testutil->test_skip_reason());
    }
  } catch (std::exception &e) {
    std::string error = e.what();
    FAIL() << makered("Unexpected exception executing test script: ")
           << e.what() << "\n";
  }
}

static bool is_identifier_assignment(std::string_view s) {
  std::string::size_type p = 0, pp;
  p = mysqlshdk::utils::span_keyword(s, p);
  if (p == 0) return false;
  p = mysqlshdk::utils::span_spaces(s, p);
  if (s[p] != '=') return false;
  ++p;
  pp = mysqlshdk::utils::span_spaces(s, p);
  pp = mysqlshdk::utils::span_keyword(s, pp);
  if (pp == p) return false;
  if (s[pp] == ';') ++pp;
  if (pp < s.size()) return false;
  return true;
}

static std::string find_in_parent_dir(std::string dir,
                                      const std::string &file) {
  while (!dir.empty() && dir != "/" && dir != "\\") {
    std::string path = shcore::path::join_path(dir, file);
    fprintf(stderr, "CHECK %s\n", path.c_str());
    if (shcore::path_exists(path)) return path;
    dir = shcore::path::dirname(dir);
  }
  return "";
}

bool Shell_script_tester::load_source_chunks(const std::string &path,
                                             std::istream &stream,
                                             const std::string &prefix) {
  std::string last_id;

  auto last_chunk = [this, &last_id]() { return &_chunks.at(last_id); };

  int linenum = 0;
  bool ret_val = true;
  bool last_chunk_enabled = true;

  while (ret_val && !stream.eof()) {
    std::string line_raw;
    std::getline(stream, line_raw);
    linenum++;

    auto line = str_rstrip_view(line_raw, "\r\n");

    auto chunk_def = load_chunk_definition(line);
    if (chunk_def.has_value()) {
      if (!prefix.empty()) {
        chunk_def->id = prefix + chunk_def->id;
        chunk_def->validation_id = prefix + chunk_def->validation_id;
      }

      // NOTE! this allows chunk_def to be moved to chunk further below
      auto chunk_is_include = str_beginswith(chunk_def->id, "INCLUDE ");

      // Full Script Context Validation is supported by defining context
      // validation at the __global__ scope.
      // The way to do it is with an anonymous chunk with the context
      // validation i.e.
      // #@ {<context validation>}
      if (chunk_def->id.empty()) {
        if ((last_id.empty() || last_id == prefix + "__global__") &&
            !context_enabled(chunk_def->context)) {
          if (output_handler.internal_std_err.empty()) {
            ADD_SKIPPED_TEST(std::string{line});
          } else {
            ADD_SKIPPED_TEST(std::string{line} + ": " +
                             output_handler.internal_std_err);
          }
          ret_val = false;
        }
      } else {
        chunk_def->linenum = linenum;

        last_id = chunk_def->id;

        // Starts the new chunk
        {
          Chunk_t chunk;
          chunk.source = path;
          chunk.def = std::exchange(*chunk_def, {});

          // Every chunk will now be added depending on the context condition
          last_chunk_enabled = add_source_chunk(path, chunk);
        }
      }

      // Handle include files... we make them look like chunks even if they
      // aren't to make it obvious the previous chunk is over
      if (chunk_is_include) {
        // Syntax:
        //@ INCLUDE file.inc
        //@ INCLUDE tag file.inc
        auto tokens = str_split(line, " ", -1, true);
        if (tokens.size() > 2) {
          std::string include = tokens[2];
          std::string tag;
          if (tokens.size() > 3) {
            tag = tokens[2];
            include = tokens[3];
          }

          // Try in the same dir as the test
          std::string inc_path =
              shcore::path::join_path(shcore::path::dirname(path), include);

          auto load = [this, include, tag](const std::string &ipath) {
            if (!ipath.empty()) {
              std::ifstream inc_stream(ipath.c_str());

              if (!inc_stream.fail()) {
                std::string namespc =
                    std::get<0>(shcore::path::split_extension(include));
                if (!tag.empty()) namespc = tag + "::" + namespc;
                load_source_chunks(include, inc_stream, namespc + "::");
                return true;
              }
            }
            return false;
          };

          if (!load(inc_path)) {
            // Try in the setup dir for the language
            std::string lang =
                std::get<1>(shcore::path::split_extension(path)).substr(1);
            if (!load(find_in_parent_dir(
                    shcore::path::dirname(path),
                    shcore::path::join_path("setup_" + lang, include)))) {
              ADD_FAILURE_AT(path.c_str(), linenum - 1)
                  << makered("Could not load include file " + inc_path) << "\n";
            }
          }
        } else {
          ADD_FAILURE_AT(path.c_str(), linenum - 1)
              << makered("Invalid INCLUDE directive") << "\n";
        }
      }
    } else if (last_chunk_enabled) {
      // Only adds the lines that are NO snippet specifier
      if (line.find("//! [") != 0) {
        if (last_id.empty()) {
          // Add __global__ at the 1st line we find
          Chunk_t chunk;
          chunk.source = path;
          chunk.def.id = last_id = prefix + "__global__";
          chunk.def.validation_id = prefix + "__global__";
          chunk.def.validation = ValidationType::Optional;
          chunk.code.push_back({linenum, std::string{line}});
          add_source_chunk(path, chunk);
        } else {
          // To simplify validation error handling and reporting, we limit
          // what can appear in an INCLUDE chunk to:
          // - comments
          // - empty space
          // - identifier assignments (x = y)
          if (str_beginswith(last_chunk()->def.id, "INCLUDE ")) {
            if (!line.empty() && !str_beginswith(line, "//") &&
                !is_identifier_assignment(line)) {
              ADD_FAILURE_AT(path.c_str(), linenum - 1)
                  << makered("No code allowed after an //@ INCLUDE directive")
                  << "\n";
            }
          }
          // If the chunk was not added because of the context, it's lines
          // are simply ignored
          last_chunk()->code.push_back({linenum, std::string{line}});
        }
      }
    }
  }

  return ret_val;
}

bool Shell_script_tester::add_source_chunk(const std::string &path,
                                           const Chunk_t &chunk) {
  bool enabled = true;
  try {
    enabled = context_enabled(chunk.def.context);
  } catch (const std::exception &) {
    // NO OP: If evaluating the context fails, it might be a context that is
    // set with the script execution so we continue considering it enabled.
    // Errors on this validation are ignored
  }

  if (enabled) {
    if (_chunks.find(chunk.def.id) == _chunks.end()) {
      _chunks[chunk.def.id] = chunk;
      // normalize Windows paths
      _chunks[chunk.def.id].source = str_replace(chunk.source, "\\", "/");
      _chunk_order.push_back(chunk.def.id);
    } else {
      ADD_FAILURE_AT(path.c_str(), chunk.def.linenum - 1)
          << makered("REDEFINITION OF CHUNK: \"") + chunk.source << ":"
          << chunk.def.line << "\"\n"
          << "\tInitially defined at " << _chunks[chunk.def.id].source << ":"
          << (_chunks[chunk.def.id].code[0].first - 1) << "\n";
    }
  } else {
    _skipped_chunks.insert(chunk.def.id);
    if (output_handler.internal_std_err.empty()) {
      SKIP_CHUNK(chunk.def.line);
    } else {
      SKIP_CHUNK(chunk.def.line + ": " + output_handler.internal_std_err);
    }
  }

  return enabled;
}

void Shell_script_tester::add_validation(Chunk_definition chunk,
                                         const std::vector<std::string> &source,
                                         std::string_view sep) {
  if (source.size() == 3) {
    auto &validations = m_chunk_validations[chunk.id];
    validations.push_back(Validation(source, std::move(chunk)));
    return;
  }

  auto text = makered("WRONG VALIDATION FORMAT FOR CHUNK ");
  text += chunk.line;
  text += "\nLine: ";
  text += shcore::str_join(source, sep);
  SCOPED_TRACE(text.c_str());
  ADD_FAILURE();
}

/**
 * Process the given line to determine if it is a chunk definition.
 * @param line The line to be processed.
 *
 * @returns The chunk definition if the line is in the right format.
 */
std::optional<Chunk_definition> Shell_script_tester::load_chunk_definition(
    std::string_view line) {
  if (line.find(get_chunk_token()) != 0) return {};

  auto chunk_id = line.substr(get_chunk_token().size());
  if (chunk_id[0] == '#') {
    if (chunk_id.size() > 1 && chunk_id[1] == ' ')
      chunk_id = chunk_id.substr(2);
    else
      chunk_id = chunk_id.substr(1);
  }

  // Identifies the version for the chunk expectations
  // If no version is specified assigns '*'
  auto start = chunk_id.find("{");
  auto end = chunk_id.find_last_of("}");

  std::string_view chunk_context;
  if (start != std::string::npos && end != std::string::npos && start < end) {
    chunk_context = chunk_id.substr(start + 1, end - start - 1);
    chunk_id = chunk_id.substr(0, start);
  }

  // Identifies the validation type and the stream if applicable
  std::string_view stream;
  ValidationType val_type{ValidationType::Simple};
  if (chunk_id.find("<OUT>") == 0) {
    stream = "OUT";
    chunk_id = chunk_id.substr(5);
    chunk_id = str_strip_view(chunk_id);
    val_type = ValidationType::Multiline;
  } else if (chunk_id.find("<ERR>") == 0) {
    stream = "ERR";
    chunk_id = chunk_id.substr(5);
    chunk_id = str_strip_view(chunk_id);
    val_type = ValidationType::Multiline;
  } else if (chunk_id.find("<PROTOCOL>") == 0) {
    stream = "PROTOCOL";
    chunk_id = chunk_id.substr(10);
    chunk_id = str_strip_view(chunk_id);
    val_type = ValidationType::Multiline;
  } else if (chunk_id.find("<>") == 0) {
    stream = "";
    chunk_id = chunk_id.substr(2);
    chunk_id = str_strip_view(chunk_id);
    val_type = ValidationType::Optional;
  }

  // Identifies the version for the chunk expectations
  // If no version is specified assigns '*'
  start = chunk_id.find("[USE:");
  end = chunk_id.find("]", start);

  std::string validation_id;
  if (start != std::string::npos && end != std::string::npos) {
    validation_id = chunk_id.substr(start + 5, end - start - 5);
    chunk_id = chunk_id.substr(0, start);
  } else {
    validation_id = chunk_id;
  }

  chunk_id = str_strip_view(chunk_id);
  validation_id = str_strip_view(validation_id);

  Chunk_definition ret_val;
  ret_val.line = line;
  ret_val.id = std::string{chunk_id};
  ret_val.context = std::string{chunk_context};
  ret_val.validation = val_type;
  ret_val.stream = std::string{stream};
  ret_val.validation_id = std::string{validation_id};

  return ret_val;
}

void Shell_script_tester::load_validations(const std::string &path) {
  m_chunk_validations.clear();

  std::ifstream file(path.c_str());
  if (file.fail()) {
    if (_new_format) {
      auto text = shcore::str_format("Unable to locate validation script: %s",
                                     path.c_str());
      SCOPED_TRACE(text.c_str());
      ADD_FAILURE();
    }

    return;
  }

  std::vector<std::string> lines;
  bool skip_chunk = false;
  bool chunk_verification = !_chunk_order.empty();
  size_t chunk_index = 0;
  Chunk_t *current_chunk = &_chunks[_chunk_order[chunk_index]];

  if (g_generate_validation_file) chunk_verification = false;

  int line_no = 0;

  std::optional<Chunk_definition> current_val_def;
  while (!file.eof()) {
    std::string line;
    std::getline(file, line);
    line_no++;

    line = shcore::str_rstrip(line);

    // If a new chunk definition is found
    // Adds the accumulated validations to the previous chunk
    auto new_val_def = load_chunk_definition(line);
    if (!new_val_def) {
      if (!skip_chunk && current_val_def.has_value()) {
        if (current_val_def->validation != ValidationType::Multiline) {
          // When processing single line validations, lines as comments are
          // ignored
          if (!shcore::str_beginswith(line, get_comment_token())) {
            line = str_strip(line);
            if (!line.empty()) {
              std::vector<std::string> tokens;
              // parse each line as:
              // <sep>stdout<sep>stderr<sep>
              // where <sep> can be any single char, such as |
              std::string sep = line.substr(0, 1);
              tokens = split_string(line, sep, false);
              add_validation(*current_val_def, tokens, sep);
            }
          }
        } else {
          lines.push_back(line);
        }
      }
      continue;
    }

    new_val_def->linenum = line_no;
    skip_chunk = false;

    // Adds the previous validations
    if (current_val_def.has_value() && lines.size()) {
      auto value = shcore::str_rstrip(multiline(lines));

      if (current_val_def->stream == "OUT" ||
          current_val_def->stream == "PROTOCOL")
        add_validation(std::move(*current_val_def), {"", std::move(value), ""});
      else if (current_val_def->stream == "ERR")
        add_validation(std::move(*current_val_def), {"", "", std::move(value)});

      current_val_def = std::move(*new_val_def);

      lines.clear();
    } else {
      current_val_def = std::move(*new_val_def);
    }

    // When the script is loaded in chunks, the validations should come in
    // the same order the chunks were loaded
    if (chunk_verification) {
      // Ensures the found validation is for a valid chunk
      if (_chunks.find(current_val_def->id) == _chunks.end() &&
          !g_generate_validation_file) {
        // The error is ONLY if the script chunk was not skipped
        if (std::find(_skipped_chunks.begin(), _skipped_chunks.end(),
                      current_val_def->id) == _skipped_chunks.end()) {
          ADD_FAILURE_AT(path.c_str(), line_no)
              << makered("FOUND VALIDATION FOR UNEXISTING CHUNK ")
              << current_val_def->line << "\n"
              << "\tLINE: " << line_no << "\n";
        }
        skip_chunk = true;
        continue;
      }

      bool match = current_val_def->id == current_chunk->def.id;

      // If the new validation no longer match the current chunk, steps
      // to the next chunk
      if (!match) {
        chunk_index++;
        current_chunk = &_chunks[_chunk_order[chunk_index]];
        match = current_val_def->id == current_chunk->def.id;
      }

      bool optional = current_chunk->is_validation_optional();
      bool reference =
          current_chunk->def.id != current_chunk->def.validation_id;

      while ((optional || reference) && !match &&
             chunk_index < _chunk_order.size()) {
        if (reference) {
          auto index =
              m_chunk_validations.find(current_chunk->def.validation_id);

          if (index == m_chunk_validations.end()) {
            ADD_FAILURE_AT(path.c_str(), line_no)
                << makered("CHUNK REFERENCES AN UNEXISTING VALIDATION")
                << current_chunk->source << ":" << current_chunk->def.line
                << "\n"
                << "\tLINE: " << line_no << "\n";
          }
        }

        chunk_index++;
        current_chunk = &_chunks[_chunk_order[chunk_index]];
        optional = current_chunk->is_validation_optional();
        reference = current_chunk->def.id != current_chunk->def.validation_id;
        match = current_val_def->id == current_chunk->def.id;
      }

      if (!optional && !match && !g_generate_validation_file) {
        ADD_FAILURE_AT(path.c_str(), line_no)
            << makered("EXPECTED VALIDATIONS FOR CHUNK ")
            << current_chunk->source << ":" << current_chunk->def.line << "\n"
            << "INSTEAD FOUND FOR CHUNK " << current_val_def->line << "\n"
            << "\tLINE: " << line_no << "\n";
        skip_chunk = true;
        continue;
      }

      if (chunk_index >= _chunk_order.size() && !g_generate_validation_file) {
        ADD_FAILURE_AT(path.c_str(), line_no)
            << makered("UNEXPECTED VALIDATIONS FOR CHUNK ")
            << current_val_def->line << "\n"
            << "\tLINE: " << line_no << "\n";
        skip_chunk = true;
      }
    }

    // If the new chunk is wrong, ignores it
    // if (!skip_chunk)
    //  current_chunk = new_val_def;
  }

  // Adds final formatted value if any
  if (current_val_def.has_value() &&
      current_val_def->validation == ValidationType::Multiline) {
    auto value = str_rstrip(multiline(lines));

    if (current_val_def->stream == "OUT" ||
        current_val_def->stream == "PROTOCOL")
      add_validation(std::move(*current_val_def), {"", value, ""});
    else if (current_val_def->stream == "ERR")
      add_validation(std::move(*current_val_def), {"", "", value});
  }

  file.close();
}

void Shell_script_tester::execute_script(const std::string &path,
                                         bool in_chunks, bool is_pre_script) {
  // If no path is provided then executes the setup script
  std::string script;
  if (path.empty())
    script = _setup_script;
  else if (is_pre_script)
    script = PRE_SCRIPT(path);
  else
    script = _new_format ? NEW_TEST_SCRIPT(path) : TEST_SCRIPT(path);

  std::ifstream stream(script.c_str());

  if (!stream.fail()) {
    TestResultCatcher catcher(&output_handler.full_output);

    // Capture GTest output if we're not tracing, so that it can be dumped
    // together with the test trace at the end.
    if (g_test_trace_scripts == 0)
      testing::UnitTest::GetInstance()->listeners().Append(&catcher);

    shcore::on_leave_scope cleaner([&catcher]() {
      if (g_test_trace_scripts == 0)
        testing::UnitTest::GetInstance()->listeners().Release(&catcher);
    });

    // When it is a test script, preprocesses it so the
    // right execution scenario is in place
    if (!path.empty()) {
      if (!is_pre_script) {
        // Processes independent preprocessing file
        std::string pre_script = PRE_SCRIPT(path);
        std::ifstream pre_stream(pre_script.c_str());
        if (!pre_stream.fail()) {
          pre_stream.close();
          _custom_context = "Preprocessing";
          execute_script(path, false, true);
        }
      }

      // Preprocesses the test file itself
      _custom_context = "Setup";
      process_setup(stream);
    }

    // Process the file
    if (in_chunks) {
      _options->interactive = true;
      if (load_source_chunks(script, stream)) {
        if (!_chunks.empty()) {
          // Loads the validations
          load_validations(_new_format ? VAL_SCRIPT(path)
                                       : VALIDATION_SCRIPT(path));
        } else {
          ADD_SKIPPED_TEST("All test chunks were skipped.");
        }
      }

      // Abort the script processing if something went wrong on the validation
      // loading
      if (testutil->test_skipped() && ::testing::Test::HasFailure()) return;

      std::ofstream ofile;
      if (g_generate_validation_file) {
        std::string vfile_name = VALIDATION_SCRIPT(path);
        if (!shcore::is_file(vfile_name)) {
          ofile.open(vfile_name, std::ofstream::out | std::ofstream::trunc);
        } else {
          vfile_name.append(".new");
          ofile.open(vfile_name, std::ofstream::out | std::ofstream::trunc);
        }
      }

      bool skip_until_cleanup = false;
      for (size_t index = 0; index < _chunk_order.size(); index++) {
        // Prints debugging information
        _cout.str("");
        _cout.clear();
        if (str_beginswith(_chunk_order[index], "INCLUDE ")) {
          const std::string &chunk_log = _chunk_order[index];
          std::string splitter(chunk_log.length(), '=');

          output_handler.debug_print(makelblue(splitter));
          output_handler.debug_print(makelblue(chunk_log));
          output_handler.debug_print(makelblue(splitter));
        } else {
          std::string chunk_log{"CHUNK: "};
          chunk_log += _chunk_order[index];
          auto splitter = makeyellow(std::string(chunk_log.length(), '-'));

          output_handler.debug_print(splitter);
          output_handler.debug_print(makeyellow(chunk_log));
          output_handler.debug_print(splitter);
        }

        // Gets the chunks for the next id
        auto &chunk = _chunks[_chunk_order[index]];

        bool enabled;
        try {
          enabled = context_enabled(chunk.def.context);

          if (enabled && skip_until_cleanup) {
            if (_chunk_order[index] == "Cleanup") {
              skip_until_cleanup = false;
            } else {
              output_handler.debug_print("Chunk skipped...");
              enabled = false;
            }
          }
        } catch (const std::exception &e) {
          ADD_FAILURE_AT(chunk.source.c_str(), chunk.code[0].first)
              << makered("ERROR EVALUATING CONTEXT: ") << e.what() << "\n"
              << "\tCHUNK: " << chunk.def.line << "\n";
          break;
        }

        // Executes the file line by line
        if (enabled) {
          _custom_context = "while executing chunk \"" + chunk.def.line +
                            "\" at " + chunk.source + ":" +
                            std::to_string(chunk.def.linenum);
          set_scripting_context();
          auto &code = chunk.code;
          std::string full_statement;
          for (size_t chunk_item = 0; chunk_item < code.size(); chunk_item++) {
            std::string line(code[chunk_item].second);

            full_statement.append(line);
            // Execution context is at line (statement actually) level
            _custom_context = chunk.source + "@[" + _chunk_order[index] + "][" +
                              std::to_string(chunk.code[chunk_item].first) +
                              ":" + full_statement + "]";

            // There's chance to do preprocessing
            pre_process_line(path, &line);

            if (testutil)
              testutil->set_test_execution_context(
                  chunk.source.c_str(), code[chunk_item].first, this);

            if (g_tdb->will_execute(chunk.source, chunk.code[chunk_item].first,
                                    line) ==
                mysqlsh::Test_debugger::Action::Skip_execute) {
              continue;
            }

            try {
              std::unique_ptr<Timeout> timeout;
              if (g_test_script_timeout > 0)
                timeout = std::make_unique<Timeout>(this);
              execute(chunk.code[chunk_item].first, line);
              if (timeout && timeout->did_timeout()) {
                ADD_FAILURE() << "line took too long: " << line
                              << "\nSkipping the rest of the script...";
                skip_until_cleanup = true;
              }
            } catch (...) {
              g_tdb->did_throw(chunk.code[chunk_item].first, line);
              throw;
            }
            if (testutil->test_skipped()) return;

            if (_interactive_shell->input_state() == shcore::Input_state::Ok)
              full_statement.clear();
            else
              full_statement.append("\n");

            g_tdb->did_execute(chunk.code[chunk_item].first, line);

            if (::testing::Test::HasFailure()) {
              if (g_tdb->did_execute_test_failure() ==
                  mysqlsh::Test_debugger::Action::Abort)
                FAIL();
            }
          }

          execute("");

          if (g_generate_validation_file) {
            // Only saves the data if the chunk is not a reference
            if (chunk.def.id == chunk.def.validation_id) {
              if (_options->trace_protocol) {
                std::string protocol_text = _cout.str();
                if (!protocol_text.empty()) {
                  ofile << get_chunk_token() << "<PROTOCOL> "
                        << _chunk_order[index] << std::endl;
                  ofile << protocol_text << std::endl;
                }
              }

              if (!output_handler.std_out.empty()) {
                ofile << get_chunk_token() << "<OUT> " << _chunk_order[index]
                      << std::endl;
                ofile << output_handler.std_out << std::endl;
              }

              if (!output_handler.std_err.empty()) {
                ofile << get_chunk_token() << "<ERR> " << _chunk_order[index]
                      << std::endl;
                ofile << output_handler.std_err << std::endl;
              }
            }
            output_handler.wipe_all();
            _cout.str("");
            _cout.clear();
          } else {
            // Validation contexts is at chunk level
            _custom_context =
                path + "@[" + _chunk_order[index] + " validation]";
            if (!validate(path, _chunk_order[index],
                          chunk.is_validation_optional())) {
              if (g_tdb->on_validate_fail(_chunk_order[index]) ==
                  mysqlsh::Test_debugger::Action::Abort) {
                FAIL();
              }
            } else {
              output_handler.wipe_debug_log();
            }
          }
        } else {
          _skipped_chunks.insert(chunk.def.id);
          if (output_handler.internal_std_err.empty()) {
            SKIP_CHUNK(chunk.def.line);
          } else {
            SKIP_CHUNK(chunk.def.line + ": " + output_handler.internal_std_err);
          }
        }
      }

      if (g_generate_validation_file) {
        ofile.close();
      }
    } else {  // !in_chunks
      _options->interactive = false;

      // Loads the validations, the validation is to exclude
      // - Loading validations for a pre_script
      // - Loading validations for a setup script (path empty)
      if (!is_pre_script && !path.empty())
        load_validations(_new_format ? VAL_SCRIPT(path)
                                     : VALIDATION_SCRIPT(path));

      // Abort the script processing if something went wrong on the validation
      // loading
      if (testutil->test_skipped() && ::testing::Test::HasFailure()) return;

      // Processes the script
      _interactive_shell->process_stream(stream, script, {}, true);

      // When path is empty it is processing a setup script
      // If an error is found it will be printed here
      if (path.empty() || is_pre_script) {
        if (!output_handler.std_err.empty()) {
          SCOPED_TRACE(output_handler.std_err);
          std::string text("Setup Script: " + _setup_script);
          SCOPED_TRACE(text.c_str());
          ADD_FAILURE();
        }

        output_handler.wipe_all();
        _cout.str("");
        _cout.clear();
      } else {
        // If processing a tets script, performs the validations over it
        _options->interactive = true;
        if (!validate(script)) {
          if (g_test_fail_early) {
            // Failure logs are printed on the fly in debug mode
            FAIL();
          }
        } else {
          output_handler.wipe_debug_log();
        }
      }
    }

    if (::testing::Test::HasFailure() && g_test_trace_scripts == 0) {
      std::cerr << makeredbg("----------vvvv Failure Log Begin vvvv----------")
                << std::endl;
      output_handler.flush_debug_log();
      std::cerr << makeredbg("----------^^^^ Failure Log End ^^^^------------")
                << std::endl;
    }

    stream.close();
  } else {
    std::string text("Unable to open test script: " + script);
    SCOPED_TRACE(text.c_str());
    ADD_FAILURE();
  }
}

// Searches for // Assumpsions: comment, if found, creates the __assumptions__
// array And processes the _assumption_script
void Shell_script_tester::process_setup(std::istream &stream) {
  while (!stream.eof()) {
    std::string line;
    std::getline(stream, line);

    if (line.find(get_assumptions_token()) != 0) break;

    // Removes the assumptions header and parses the rest
    auto tokens = split_string(line, ":", true);

    // Now parses the real assumptions
    tokens = split_string(tokens[1], ",", true);

    // Now quotes the assumptions
    for (auto &token : tokens) {
      auto token_stripped = str_strip_view(token);
      token =
          shcore::str_format("'%.*s'", static_cast<int>(token_stripped.size()),
                             token_stripped.data());
    }

    // Creates an assumptions array to be processed on the setup script
    auto prefix = get_variable_prefix();
    auto code = shcore::str_format(
        "%.*s__assumptions__ = [%s];", static_cast<int>(prefix.size()),
        prefix.data(), str_join(tokens, ",").c_str());

    execute(code);

    if (_setup_script.empty())
      throw std::logic_error(
          "A setup script must be specified when there are assumptions on "
          "the tested scripts.");

    execute_script();  // Executes the active setup script
  }

  // Once the assumptions are processed, rewinds the read position
  // To the beggining of the script
  stream.clear();  // To clean up the eof flag in case it was set
  stream.seekg(0, stream.beg);
}

void Shell_script_tester::validate_batch(const std::string &script) {
  _filename = script;
  execute_script(script, false);
}

void Shell_script_tester::def_var(std::string_view var,
                                  std::string_view value) {
  auto prefix = get_variable_prefix();
  auto code = shcore::str_format("%.*s%.*s=%.*s",  //
                                 static_cast<int>(prefix.size()), prefix.data(),
                                 static_cast<int>(var.size()), var.data(),
                                 static_cast<int>(value.size()), value.data());

  exec_and_out_equals(code);
}

void Shell_script_tester::def_string_var_from_env(const std::string &var,
                                                  const std::string &env_var) {
  const char *variable =
      getenv(env_var.empty() ? var.c_str() : env_var.c_str());
  if (variable) {
    def_var(var, shcore::str_format("'%s'", variable));
  }
}

void Shell_script_tester::def_numeric_var_from_env(const std::string &var,
                                                   const std::string &env_var) {
  const char *variable =
      getenv(env_var.empty() ? var.c_str() : env_var.c_str());
  if (variable) {
    def_var(var, variable);
  }
}

void Shell_script_tester::set_defaults() {
  output_handler.wipe_all();
  _cout.str("");
  _cout.clear();

  Crud_test_wrapper::set_defaults();

  std::string test_mode;
  switch (g_test_recording_mode) {
    case mysqlshdk::db::replay::Mode::Direct:
      test_mode = "direct";
      break;
    case mysqlshdk::db::replay::Mode::Record:
      test_mode = "record";
      break;
    case mysqlshdk::db::replay::Mode::Replay:
      test_mode = "replay";
      break;
  }

  auto var_prefix = std::string{get_variable_prefix()};
  auto version_num = static_cast<int64_t>(_target_server_version.numeric());

  auto code = shcore::str_format("%s__test_execution_mode = '%s'",
                                 var_prefix.c_str(), test_mode.c_str());
  exec_and_out_equals(code);

  code = shcore::str_format("%s__package_year = '" PACKAGE_YEAR "'",
                            var_prefix.c_str());
  exec_and_out_equals(code);

  code =
      shcore::str_format("%s__mysh_version = '%s'", var_prefix.c_str(),
                         mysqlshdk::utils::k_shell_version.get_base().c_str());
  exec_and_out_equals(code);

  code =
      shcore::str_format("%s__mysh_version_full = '%s'", var_prefix.c_str(),
                         mysqlshdk::utils::k_shell_version.get_full().c_str());
  exec_and_out_equals(code);

  code =
      shcore::str_format("%s__mysh_version_num = %" PRIu32, var_prefix.c_str(),
                         mysqlshdk::utils::k_shell_version.numeric());
  exec_and_out_equals(code);

  code = shcore::str_format("%s__version = '%s'", var_prefix.c_str(),
                            _target_server_version.get_base().c_str());
  exec_and_out_equals(code);

  code = shcore::str_format("%s__version_full = '%s'", var_prefix.c_str(),
                            _target_server_version.get_full().c_str());
  exec_and_out_equals(code);

  code = shcore::str_format("%s__version_num = %" PRId64, var_prefix.c_str(),
                            version_num);
  exec_and_out_equals(code);

  // Set terminology related variables
  if (version_num > 80025) {
    def_var("__replica_keyword", "'replica'");
    def_var("__replica_keyword_capital", "'Replica'");
    def_var("__source_keyword", "'source'");
    def_var("__source_keyword_capital", "'Source'");
  } else {
    def_var("__replica_keyword", "'slave'");
    def_var("__replica_keyword_capital", "'Slave'");
    def_var("__source_keyword", "'master'");
    def_var("__source_keyword_capital", "'Master'");
  }

  def_var("__mysqluripwd", "''");
  def_var("__os_type", "'" + shcore::to_string(shcore::get_os_type()) + "'");
  def_var("__machine_type", "'" + shcore::get_machine_type() + "'");
  def_var("__test_data_path",
          shcore::quote_string(shcore::path::join_path(g_test_home, "data", ""),
                               '\'') +
              ";");
  def_var("__test_home_path", shcore::quote_string(g_test_home, '\'') + ";");

  if (_target_server_version >= mysqlshdk::utils::Version(8, 0, 21)) {
    def_var("__default_gr_expel_timeout", "5");
    def_var("__default_gr_auto_rejoin_tries", "3");
  } else {
    def_var("__default_gr_expel_timeout", "0");
    def_var("__default_gr_auto_rejoin_tries", "0");
  }

  if (mysqlsh::dba::supports_paxos_single_leader(_target_server_version)) {
    def_var("__default_gr_paxos_single_leader", "'OFF'");
  }

  def_var("__user_config_path",
          shcore::quote_string(shcore::get_user_config_path(), '\''));

#ifdef PYTHON_DEPS
  def_var("__python_deps", "1");
#else
  def_var("__python_deps", "0");
#endif

#ifdef NDEBUG
  // TODO(.) - remove __dbug_off and replace all uses with __dbug
  def_var("__dbug_off", "1");
  // dbug tests should only run in direct mode, so that traces aren't affected
  // by different code branches being taken
  def_var("__dbug", !_replaying && !_recording ? "1==1" : "0==1");
#else
  def_var("__dbug_off", "0");
  def_var("__dbug", "0==1");
#endif
  // Variables for OCI Tests
  def_string_var_from_env("OCI_CONFIG_HOME");
  def_string_var_from_env("OCI_COMPARTMENT_ID");
  def_string_var_from_env("OS_NAMESPACE");
  def_string_var_from_env("OS_BUCKET_NAME");

  // Variables for MDS Tests
  def_string_var_from_env("MDS_URI");
  def_string_var_from_env("MDS_LH_URI");
  def_string_var_from_env("MDS_LH_BUCKET");

  auto set_env_if_missing = [](const char *var, const char *value) {
    if (!getenv(var)) {
      shcore::setenv(var, value);
    }
  };

  // Simple LDAP Authentication Variables
  if (getenv("LDAP_SIMPLE_SERVER_HOST")) {
    set_env_if_missing("LDAP_SIMPLE_BIND_BASE_DN", "dc=my-domain,dc=com");
  }

  def_string_var_from_env("LDAP_SIMPLE_SERVER_HOST");
  def_string_var_from_env("LDAP_SIMPLE_SERVER_PORT");
  def_string_var_from_env("LDAP_SIMPLE_BIND_BASE_DN");
  def_string_var_from_env("LDAP_SIMPLE_USER");
  def_string_var_from_env("LDAP_SIMPLE_PWD");
  def_string_var_from_env("LDAP_SIMPLE_AUTH_STRING");

  // LDAP SASL Authentication Variables
  if (getenv("LDAP_SASL_SERVER_HOST")) {
    set_env_if_missing("LDAP_SASL_BIND_BASE_DN", "dc=my-domain,dc=com");
    set_env_if_missing("LDAP_SASL_GROUP_SEARCH_FILTER",
                       "(|(&(objectClass=posixGroup)(memberUid={UA}))(&("
                       "objectClass=group)(member={UD})))");
  }

  def_string_var_from_env("LDAP_SASL_SERVER_HOST");
  def_string_var_from_env("LDAP_SASL_SERVER_PORT");
  def_string_var_from_env("LDAP_SASL_BIND_BASE_DN");
  def_string_var_from_env("LDAP_SASL_USER");
  def_string_var_from_env("LDAP_SASL_PWD");
  def_string_var_from_env("LDAP_SASL_GROUP_SEARCH_FILTER");

  // LDAP Kerberos Authentication Variables
  if (getenv("LDAP_KERBEROS_SERVER_HOST")) {
    set_env_if_missing("LDAP_KERBEROS_BIND_BASE_DN",
                       "CN=users,DC=mtr,DC=local");
    set_env_if_missing("LDAP_KERBEROS_USER_SEARCH_ATTR", "sAMAccountName");
    set_env_if_missing("LDAP_KERBEROS_BIND_ROOT_DN",
                       "CN=test2,CN=Users,DC=mtr,DC=local");
    set_env_if_missing("LDAP_KERBEROS_GROUP_SEARCH_FILTER",
                       "(&(objectClass=group)(member={UD}))");
  }

  def_string_var_from_env("LDAP_KERBEROS_SERVER_HOST");
  def_string_var_from_env("LDAP_KERBEROS_SERVER_PORT");
  def_string_var_from_env("LDAP_KERBEROS_BIND_BASE_DN");
  def_string_var_from_env("LDAP_KERBEROS_USER_SEARCH_ATTR");
  def_string_var_from_env("LDAP_KERBEROS_BIND_ROOT_DN");
  def_string_var_from_env("LDAP_KERBEROS_BIND_ROOT_PWD");
  def_string_var_from_env("LDAP_KERBEROS_USER");
  def_string_var_from_env("LDAP_KERBEROS_PWD");
  def_string_var_from_env("LDAP_KERBEROS_AUTH_STRING");
  def_string_var_from_env("LDAP_KERBEROS_GROUP_SEARCH_FILTER");
  def_string_var_from_env("LDAP_KERBEROS_DOMAIN");

  // Kerberos Authentication Variables
  def_string_var_from_env("KERBEROS_USER");
  def_string_var_from_env("KERBEROS_PWD");
  def_string_var_from_env("KERBEROS_DOMAIN");

  // OCI Authentication Variables
  def_string_var_from_env("OCI_AUTH_URI");
  if (getenv("OCI_AUTH_CONFIG_FILE")) {
    def_string_var_from_env("OCI_AUTH_CONFIG_FILE");
  } else {
    def_var("OCI_AUTH_CONFIG_FILE", "''");
  }
  if (getenv("OCI_AUTH_PROFILE")) {
    def_string_var_from_env("OCI_AUTH_PROFILE");
  } else {
    def_var("OCI_AUTH_PROFILE", "'DEFAULT'");
  }
  if (getenv("OCI_AUTH_POSITIVE_TESTS")) {
    def_numeric_var_from_env("OCI_AUTH_POSITIVE_TESTS");
  } else {
    def_var("OCI_AUTH_POSITIVE_TESTS",
            getenv("OCI_AUTH_CONFIG_FILE") ? "1" : "0");
  }

  // Variables for AWS Tests
  def_string_var_from_env("MYSQLSH_S3_BUCKET_NAME");
  def_string_var_from_env("MYSQLSH_AWS_SHARED_CREDENTIALS_FILE");
  def_string_var_from_env("MYSQLSH_AWS_CONFIG_FILE");
  def_string_var_from_env("MYSQLSH_AWS_PROFILE");
  def_string_var_from_env("MYSQLSH_AWS_REGION");
  def_string_var_from_env("MYSQLSH_AWS_ROLE");
  def_string_var_from_env("MYSQLSH_S3_ENDPOINT_OVERRIDE");

  def_var("__libmysql_version_id",
          shcore::str_format("'%d'", LIBMYSQL_VERSION_ID));
}

void Shell_js_script_tester::set_defaults() {
  _interactive_shell->process_line("\\js");
  Shell_script_tester::set_defaults();

#ifdef HAVE_JS
  exec_and_out_equals("var __have_javascript = true");
#else
  exec_and_out_equals("var __have_javascript = false");
#endif
}

void Shell_py_script_tester::set_defaults() {
  _interactive_shell->process_line("\\py");
  Shell_script_tester::set_defaults();

#ifdef HAVE_JS
  exec_and_out_equals("__have_javascript = True");
#else
  exec_and_out_equals("__have_javascript = False");
#endif
}

std::string_view Shell_script_tester::get_current_mode_command() const {
  switch (_interactive_shell->shell_context()->interactive_mode()) {
    case shcore::IShell_core::Mode::SQL:
      return "\\sql";
    case shcore::IShell_core::Mode::JavaScript:
      return "\\js";
    case shcore::IShell_core::Mode::Python:
      return "\\py";
    case shcore::IShell_core::Mode::None:
      return "";
  }

  return "";
}

void Shell_script_tester::set_scripting_context() {
  auto current = get_current_mode_command();
  auto required = get_switch_mode_command();

  std::string code;
  {
    auto context = context_identifier();
    context = shcore::str_replace(context, "'", "\\'");

    auto var_prefix = get_variable_prefix();
    code = shcore::str_format("%.*s__test_context = '%s';",
                              static_cast<int>(var_prefix.size()),
                              var_prefix.data(), context.c_str());
  }

  if (current != required) {
    execute_internal(std::string{required});
    execute_internal(code);
    execute_internal(std::string{current});
  } else {
    execute_internal(code);
  }
}

void Shell_script_tester::execute_setup() {
  // No need to process setup scripts line by line
#ifdef _WIN32
  std::ifstream s(shcore::utf8_to_wide(_setup_script));
#else
  std::ifstream s(_setup_script.c_str());
#endif

  if (!s.fail()) {
    // The return value now depends on the stream processing
    _interactive_shell->process_stream(s, _setup_script, {}, true);

    s.close();
  } else {
    std::string text("Unable to open test script: " + _setup_script);
    SCOPED_TRACE(text.c_str());
    ADD_FAILURE();
  }
}

// Append option to the end of the given config file.
void Shell_script_tester::add_to_cfg_file(const std::string &cfgfile_path,
                                          const std::string &option) {
  std::ofstream cfgfile(cfgfile_path, std::ios_base::app);
  cfgfile << option << std::endl;
  cfgfile.close();
}

// Delete lines with the option from the given config file.
void Shell_script_tester::remove_from_cfg_file(const std::string &cfgfile_path,
                                               const std::string &option) {
  std::string new_cfgfile_path = cfgfile_path + ".new";
  std::ofstream new_cfgfile(new_cfgfile_path);
  std::ifstream cfgfile(cfgfile_path);
  std::string line;
  while (std::getline(cfgfile, line)) {
    if (line.find(option) != 0) new_cfgfile << line << std::endl;
  }
  cfgfile.close();
  new_cfgfile.close();
  std::remove(cfgfile_path.c_str());
  std::rename(new_cfgfile_path.c_str(), cfgfile_path.c_str());
}

// Check whether openssl executable is accessible via PATH
bool Shell_script_tester::has_openssl_binary() {
  const char *argv[] = {"openssl", "version", nullptr};
  shcore::Process_launcher p(argv);
  p.start();
  std::string s = p.read_line();
  if (p.wait() == 0) {
    return shcore::str_beginswith(s, "OpenSSL");
  }
  return false;
}

bool Shell_script_tester::context_enabled(std::string code) {
  if (code.empty()) return true;

  auto function_pos = code.find("VER(");
  while (function_pos != std::string::npos) {
    size_t version_pos = code.find_first_of("0123456789", function_pos);
    size_t closing_pos = code.find(")", version_pos);

    if (version_pos == std::string::npos || closing_pos == std::string::npos)
      throw std::invalid_argument("Invalid syntax for VER(#.#.#) macro.");

    std::string old_func =
        code.substr(function_pos, closing_pos - function_pos + 1);
    std::string op = shcore::str_strip(
        code.substr(function_pos + 4, version_pos - function_pos - 4));
    std::string ver =
        shcore::str_strip(code.substr(version_pos, closing_pos - version_pos));
    std::string fname =
        shcore::get_member_name("versionCheck", get_naming_style());
    std::string new_func =
        "testutil." + fname + "(__version, '" + op + "', '" + ver + "')";

    code = shcore::str_replace(code, old_func, new_func);
    function_pos = code.find("VER(");
  }

  function_pos = code.find("DEF(");
  while (function_pos != std::string::npos) {
    size_t closing_pos = code.find(")", function_pos);
    if (closing_pos == std::string::npos)
      throw std::invalid_argument("Invalid syntax for DEF(variable) macro.");

    std::string old_func =
        code.substr(function_pos, closing_pos - function_pos + 1);
    std::string variable = shcore::str_strip(
        code.substr(function_pos + 4, closing_pos - function_pos - 4));

    std::string new_func = get_if_def(variable);
    code = shcore::str_replace(code, old_func, new_func);
    function_pos = code.find("DEF(");
  }

  output_handler.wipe_out();

  execute_internal(code);

  {
    auto value = str_strip_view(output_handler.internal_std_out);
    if (value == "true" || value == "True") return true;
    if (value == "false" || value == "False") return false;
  }

  if (!output_handler.internal_std_err.empty()) {
    throw std::invalid_argument(output_handler.internal_std_err);
  }

  throw std::invalid_argument(
      "Context does not evaluate to a boolean expression");
}

void Shell_script_tester::execute(int location, const std::string &code) {
  // save location in test script that is being currently executed
  _current_entry_point = context_identifier();
  try {
    Crud_test_wrapper::execute(location, code);
    _current_entry_point.clear();
  } catch (...) {
    _current_entry_point.clear();
    throw;
  }
}

void Shell_script_tester::execute(const std::string &code) {
  // save location in test script that is being currently executed
  _current_entry_point = context_identifier();
  try {
    Crud_test_wrapper::execute(code);
    _current_entry_point.clear();
  } catch (...) {
    _current_entry_point.clear();
    throw;
  }
}
